package com.coderside.fragment.netWork;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.HttpClient;

/**
 * 爬取并分析bling0网站的文章
 * 并存储在数据中，数据库建表脚本在article.sql
 * @author admin
 *
 */
public class ParseQiubai {
	private static String defaultEncode = "UTF-8";

	private static String dirPath = "D:\\qiushibaike";
	private static String baseUrl = "http://www.qiushibaike.com";
	private static Map<String, Pattern> patternMap = new HashMap<String, Pattern>();

	/**
	* 测试
	*
	* @param args
	*/
	public static void main(String[] args) {
		//System.setProperty("http.proxyHost", "165.228.128.10");
		//System.setProperty("http.proxyPort", "3128");
		//System.setProperty("http.proxySet", "true");

		/***当你爬取的内容过于频繁的时候、Iteye会要求输入一个验证码，导致运行异常**/
		String reqUrl = null;
		long a = System.currentTimeMillis();
		for (int i = 1; i <= 2; i++) {
			//			reqUrl = "http://bling0.com/all/latest/page/" + i;
			reqUrl = "http://www.qiushibaike.com/new2/month/20/page/" + i;
			parsePage(reqUrl);
		}
		long b = System.currentTimeMillis();
		System.out.println(b - a);
	}

	/**
	 * 保存正则表达式到缓存中并返回
	 * 下次可以直接返回 不用重复编译
	 * PATTERNMODE 为正则表达式的模式参数 e.g:Pattern.CASE_INSENSITIVE
	 * @param regex
	 * @param PATTERNMODE 为-1的时候说明不需要该参数
	 * @return
	 */
	public static Pattern cachePattern(String regex, int PATTERNMODE) {
		Pattern pattern = patternMap.get(regex);
		if (null != pattern) {
			return pattern;
		}
		Pattern p = null;
		if (PATTERNMODE == -1) {
			p = Pattern.compile(regex);
			patternMap.put(regex, p);
		} else {
			p = Pattern.compile(regex, PATTERNMODE);
			patternMap.put(regex, p);
		}
		return p;
	}

	/**
	 * 去掉所有html标签
	 * @param html
	 * @return
	 */
	public static String replaceTag(String html) {
		String regEx = "<.+?>"; //表示标签   
		Pattern p = Pattern.compile(regEx, Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(html);
		//		System.out.println(m.find());
		String text = m.replaceAll("");
		text = text.trim();
		text = text.replace("　　", "");
		//		System.out.println(text.replace("　　", ""));
		return text;
	}

	/**
	 * 
	 * @param html
	 */
	public static String parseUrl(String html) {
		//		<a href="/new2/article/3127398" target="_blank"></a>
		String regex = "<a href=\"(.+?)\" target=\"_blank\">(.*?)</a>";
		//在 dotall 模式中，表达式 . 可以匹配任何字符，包括行结束符。默认情况下，此表达式不匹配行结束符。
		Pattern p = Pattern.compile(regex, Pattern.DOTALL);
		Matcher matcher = null;
		matcher = p.matcher(html);
		if (matcher != null && matcher.find()) {
			return matcher.group(1);
		}
		return "";
	}

	/**
	 * 每个article有title、up、down、 content、comments、relativetime,tag
	 * 使用正则多行匹配article标签内的内容
	 * @param body
	 */
	public static ArticleQiubai pickupArticle(String articleHtml, ArticleQiubai article) {
		pickupContentAndTime(articleHtml, article);
		pickupUp(articleHtml, article);
		pickupDown(articleHtml, article);
		pickupTag(articleHtml, article);
		pickupUrl(articleHtml, article);
		return article;
	}

	private static void pickupUrl(String articleHtml, ArticleQiubai article) {
		// TODO Auto-generated method stub

		//		<div class="detail">
		String regex = "(?i)<div class=\"detail\">([.+\\s\\S*]+?)</div>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<a href="/truth" class="group" data-alias="truth">博聆真相调查组</a> 提取"博聆真相调查组"
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String url = m.group(1);
			url = parseUrl(url);
			article.setUrl(url);
		}

	}

	private static void pickupTag(String articleHtml, ArticleQiubai article) {
		//		<div class="tags">
		String regex = "(?i)<div class=\"tags\">([.+\\s\\S*]+?)</div>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<a href="/truth" class="group" data-alias="truth">博聆真相调查组</a> 提取"博聆真相调查组"
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String tag = m.group(1);
			article.setTag(tag);
		}
	}

	/**
	 * 解析文章的支持数
	 * @param articleHtml
	 * @param article
	 */
	private static void pickupUp(String articleHtml, ArticleQiubai article) {

		//		<div class="up" id="vote-up-3186653">
		String regex = "(?i)<div class=\"up\" id=\"(.+?)\">([.+\\s\\S*]+?)</div>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<div class='entry-content'>开头，</div>结尾的文档
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String uphtml = m.group(2);
			String upNum = replaceTag(uphtml);
			article.setUp(upNum);
		}

	}

	/**
	 * 解析文章的反对数
	 * @param articleHtml
	 * @param article
	 */
	private static void pickupDown(String articleHtml, ArticleQiubai article) {

		//		<div class="up" id="vote-up-3186653">
		String regex = "(?i)<div class=\"down\" id=\"(.+?)\"\\s>([.+\\s\\S*]+?)</div>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<div class='entry-content'>开头，</div>结尾的文档
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String uphtml = m.group(2);
			String downNum = replaceTag(uphtml);
			article.setDown(downNum);
		}

	}

	private static void pickupContentAndTime(String articleHtml, ArticleQiubai article) {
		//		<div class="content" title="2012-03-31 14:47:21">
		String regex = "(?i)<div class=\"content\" title=\"(.+?)\">([.+\\s\\S*]+?)</div>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<div class='entry-content'>开头，</div>结尾的文档
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String relativeTime = m.group(1);
			article.setRelativetime(relativeTime);
			String contentHtml = m.group(2);
			if (hasTag("img", contentHtml)) {
				pickupImage(contentHtml, article);
			}
			String content = replaceTag(contentHtml);
			article.setContent(content);
			regex = "[.+\\s\\S*]+?\\(剩余\\d{1,5}字\\)";
			if (matcherReg(regex, content) && null != article.getUrl()) {
				String articleUrl = baseUrl + article.getUrl();
				System.out.println(articleUrl);
				parseItem(articleUrl, article);
			}

		}
	}

	private static void pickupItemContent(String articleHtml, ArticleQiubai article) {
		String regex = "(?i)<div class=\'entry-content\'>([.+\\s\\S*]+?)</div>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<div class='entry-content'>开头，</div>结尾的文档
		Matcher m = p.matcher(articleHtml);//开始编译
		if (m.find()) {
			String contentHtml = m.group(1);
			String content = replaceTag(contentHtml);
			article.setContent(content);
		}
	}

	private static void pickupImage(String articleHtml, ArticleQiubai article) {
		//		String exampleText = "<div class=\"entry-content\"><div class=\"img_link\"><a href=\"/system/pictures/458649/large/%E7%9C%BC%E9%95%9C%E5%BE%88%E9%87%8D%E8%A6%81.jpg?1333322359\" class=\"picture\" id=\"picture-458649\" title=\"事实证明，包装很重要，特别是眼镜.....\"><img alt=\"\" src=\"/system/pictures/458649/thumb/%E7%9C%BC%E9%95%9C%E5%BE%88%E9%87%8D%E8%A6%81.jpg?1333322359\"></a></div><p></p></div>";
		String regex = "<img.+src=\"(.+)\".+?>"; //表示标签   
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<table .+?>开头，</table>结尾的文档
		Matcher m = p.matcher(articleHtml);
		String localSmallPath = dirPath + "\\" + "\\" + article.getId() + "_small.jpg";
		String localLargePath = dirPath + "\\" + "\\" + article.getId() + "_large.jpg";
		article.setLocalImg(localSmallPath);
		if (new File(localSmallPath).exists()) {
			return;
		}
		if (m != null && m.find()) {
			String remoteThumbFile = baseUrl + m.group(1);
			String remoteLargeFile = remoteThumbFile.replace("thumb", "large");
			new CopyFile().copyFile(localSmallPath, remoteThumbFile);
			new CopyFile().copyFile(localLargePath, remoteLargeFile);
		}
	}

	public static boolean matcherReg(String regEx, String content) {
		Pattern p = cachePattern(regEx, Pattern.MULTILINE);
		Matcher m = p.matcher(content);
		return m.find();
	}

	/**
	 * 判断是否包含有html标签
	 * @param html
	 * @return
	 */
	public static boolean hasTag(String tag, String html) {
		String regEx = "<" + tag; //表示标签   
		Pattern p = Pattern.compile(regEx, Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(html);
		return m.find();
	}

	/**
	 * 每页有25条<article></article>
	 * 使用正则多行匹配table标签内的内容
	 * @param body
	 */
	public static List<ArticleQiubai> pickupArticlesList(String body) {
		//		<div class="block untagged" id='qiushi_tag_3200733'>
		String regex = "(?i)<div class=\"block untagged\" id='qiushi_tag_(\\d+?)'>([.+\\s\\S*]+?)<div class=\"shadow\"></div>";
		List<ArticleQiubai> articles = new ArrayList<ArticleQiubai>();
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<table .+?>开头，</table>结尾的文档
		Matcher m = p.matcher(body);//开始编译
		while (m.find()) {
			String articleId = m.group(1);
			ArticleQiubai article = new ArticleQiubai();
			article.setId(articleId);
			pickupArticle(m.group(2), article);
			articles.add(article);
		}
		return articles;
	}

	/**
	 * 解析论坛主页 获取主页上的链接
	 * @param reqUrl
	 */
	public static void parsePage(String pageUrl) {

		//System.setProperty("http.proxyHost", "165.228.128.10");
		//System.setProperty("http.proxyPort", "3128");
		//System.setProperty("http.proxySet", "true");

		//		reqUrl = "http://www.iteye.com/forums/tag/Java";
		Map<String, String> headers = new HashMap<String, String>();
		headers.put("Accept-Encoding", "gzip,deflate");

		HttpClient httpClient = HttpClientHelper.getHttpClient();
		String htmlContent = HttpClientHelper.doRequest(httpClient, pageUrl, headers, null, defaultEncode);
		List<ArticleQiubai> articles = pickupArticlesList(htmlContent);
		//		DBTool.insertArticles(articles);
	}

	/**
	 * 解析论坛主页 获取主页上的链接
	 * @param reqUrl
	 */
	public static void parseItem(String articleUrl, ArticleQiubai article) {

		//System.setProperty("http.proxyHost", "165.228.128.10");
		//System.setProperty("http.proxyPort", "3128");
		//System.setProperty("http.proxySet", "true");

		//		reqUrl = "http://www.iteye.com/forums/tag/Java";
		Map<String, String> headers = new HashMap<String, String>();
		//		headers.put("Accept-Encoding", "gzip,deflate");

		HttpClient httpClient = HttpClientHelper.getHttpClient();
		String htmlContent = HttpClientHelper.doRequest(httpClient, articleUrl, headers, null, defaultEncode);
		pickupItemContent(htmlContent, article);
	}

}
